


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.optim.lr_scheduler &mdash; PyTorch master documentation</title>
  

  
  
  
  
    <link rel="canonical" href="https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html"/>
  

  

  
  
    

  

  <link rel="stylesheet" href="../../../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../../../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../../../genindex.html" />
    <link rel="search" title="Search" href="../../../search.html" /> 

  
  <script src="../../../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../../../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  master (1.5.0 )
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
<div>
  <a style="color:#F05732" href="https://pytorch.org/docs/stable/_modules/torch/optim/lr_scheduler.html">
    You are viewing unstable developer preview docs.
    Click here to view docs for latest stable release.
  </a>
</div>

            
            
              
            
            
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../packages.html">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../amp.html">torch.cuda.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../rpc/index.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
          

        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../../../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
          <li><a href="../../index.html">Module code</a> &gt;</li>
        
          <li><a href="../../torch.html">torch</a> &gt;</li>
        
      <li>torch.optim.lr_scheduler</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <h1>Source code for torch.optim.lr_scheduler</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">types</span>
<span class="kn">import</span> <span class="nn">math</span>
<span class="kn">from</span> <span class="nn">torch._six</span> <span class="kn">import</span> <span class="n">inf</span>
<span class="kn">from</span> <span class="nn">functools</span> <span class="kn">import</span> <span class="n">wraps</span>
<span class="kn">import</span> <span class="nn">warnings</span>
<span class="kn">import</span> <span class="nn">weakref</span>
<span class="kn">from</span> <span class="nn">collections</span> <span class="kn">import</span> <span class="n">Counter</span>
<span class="kn">from</span> <span class="nn">bisect</span> <span class="kn">import</span> <span class="n">bisect_right</span>

<span class="kn">from</span> <span class="nn">.optimizer</span> <span class="kn">import</span> <span class="n">Optimizer</span>


<span class="n">EPOCH_DEPRECATION_WARNING</span> <span class="o">=</span> <span class="p">(</span>
    <span class="s2">&quot;The epoch parameter in `scheduler.step()` was not necessary and is being &quot;</span>
    <span class="s2">&quot;deprecated where possible. Please use `scheduler.step()` to step the &quot;</span>
    <span class="s2">&quot;scheduler. During the deprecation, if epoch is different from None, the &quot;</span>
    <span class="s2">&quot;closed form is used instead of the new chainable form, where available. &quot;</span>
    <span class="s2">&quot;Please open an issue if you are unable to replicate your use case: &quot;</span>
    <span class="s2">&quot;https://github.com/pytorch/pytorch/issues/new/choose.&quot;</span>
<span class="p">)</span>

<span class="n">SAVE_STATE_WARNING</span> <span class="o">=</span> <span class="s2">&quot;Please also save or load the state of the optimzer when saving or loading the scheduler.&quot;</span>

<span class="k">class</span> <span class="nc">_LRScheduler</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>

        <span class="c1"># Attach optimizer</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">Optimizer</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> is not an Optimizer&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="nb">type</span><span class="p">(</span><span class="n">optimizer</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optimizer</span>

        <span class="c1"># Initialize epoch and base learning rates</span>
        <span class="k">if</span> <span class="n">last_epoch</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">:</span>
                <span class="n">group</span><span class="o">.</span><span class="n">setdefault</span><span class="p">(</span><span class="s1">&#39;initial_lr&#39;</span><span class="p">,</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">])</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                <span class="k">if</span> <span class="s1">&#39;initial_lr&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">group</span><span class="p">:</span>
                    <span class="k">raise</span> <span class="ne">KeyError</span><span class="p">(</span><span class="s2">&quot;param &#39;initial_lr&#39; is not specified &quot;</span>
                                   <span class="s2">&quot;in param_groups[</span><span class="si">{}</span><span class="s2">] when resuming an optimizer&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">i</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">group</span><span class="p">:</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;initial_lr&#39;</span><span class="p">],</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">=</span> <span class="n">last_epoch</span>

        <span class="c1"># Following https://github.com/pytorch/pytorch/issues/20124</span>
        <span class="c1"># We would like to ensure that `lr_scheduler.step()` is called after</span>
        <span class="c1"># `optimizer.step()`</span>
        <span class="k">def</span> <span class="nf">with_counter</span><span class="p">(</span><span class="n">method</span><span class="p">):</span>
            <span class="k">if</span> <span class="nb">getattr</span><span class="p">(</span><span class="n">method</span><span class="p">,</span> <span class="s1">&#39;_with_counter&#39;</span><span class="p">,</span> <span class="kc">False</span><span class="p">):</span>
                <span class="c1"># `optimizer.step()` has already been replaced, return.</span>
                <span class="k">return</span> <span class="n">method</span>

            <span class="c1"># Keep a weak reference to the optimizer instance to prevent</span>
            <span class="c1"># cyclic references.</span>
            <span class="n">instance_ref</span> <span class="o">=</span> <span class="n">weakref</span><span class="o">.</span><span class="n">ref</span><span class="p">(</span><span class="n">method</span><span class="o">.</span><span class="vm">__self__</span><span class="p">)</span>
            <span class="c1"># Get the unbound method for the same purpose.</span>
            <span class="n">func</span> <span class="o">=</span> <span class="n">method</span><span class="o">.</span><span class="vm">__func__</span>
            <span class="bp">cls</span> <span class="o">=</span> <span class="n">instance_ref</span><span class="p">()</span><span class="o">.</span><span class="vm">__class__</span>
            <span class="k">del</span> <span class="n">method</span>

            <span class="nd">@wraps</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
            <span class="k">def</span> <span class="nf">wrapper</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
                <span class="n">instance</span> <span class="o">=</span> <span class="n">instance_ref</span><span class="p">()</span>
                <span class="n">instance</span><span class="o">.</span><span class="n">_step_count</span> <span class="o">+=</span> <span class="mi">1</span>
                <span class="n">wrapped</span> <span class="o">=</span> <span class="n">func</span><span class="o">.</span><span class="fm">__get__</span><span class="p">(</span><span class="n">instance</span><span class="p">,</span> <span class="bp">cls</span><span class="p">)</span>
                <span class="k">return</span> <span class="n">wrapped</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>

            <span class="c1"># Note that the returned function here is no longer a bound method,</span>
            <span class="c1"># so attributes like `__func__` and `__self__` no longer exist.</span>
            <span class="n">wrapper</span><span class="o">.</span><span class="n">_with_counter</span> <span class="o">=</span> <span class="kc">True</span>
            <span class="k">return</span> <span class="n">wrapper</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span> <span class="o">=</span> <span class="n">with_counter</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">_step_count</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_step_count</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">step</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Returns the state of the scheduler as a :class:`dict`.</span>

<span class="sd">        It contains an entry for every variable in self.__dict__ which</span>
<span class="sd">        is not the optimizer.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="n">value</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">key</span> <span class="o">!=</span> <span class="s1">&#39;optimizer&#39;</span><span class="p">}</span>

    <span class="k">def</span> <span class="nf">load_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dict</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Loads the schedulers state.</span>

<span class="sd">        Arguments:</span>
<span class="sd">            state_dict (dict): scheduler state. Should be an object returned</span>
<span class="sd">                from a call to :meth:`state_dict`.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">state_dict</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_last_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot; Return last computed learning rate by current scheduler.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_last_lr</span>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="c1"># Compute learning rate using chainable form of the scheduler</span>
        <span class="k">raise</span> <span class="ne">NotImplementedError</span>

    <span class="k">def</span> <span class="nf">step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">epoch</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="c1"># Raise a warning if old pattern is detected</span>
        <span class="c1"># https://github.com/pytorch/pytorch/issues/20124</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_step_count</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">step</span><span class="p">,</span> <span class="s2">&quot;_with_counter&quot;</span><span class="p">):</span>
                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Seems like `optimizer.step()` has been overridden after learning rate scheduler &quot;</span>
                              <span class="s2">&quot;initialization. Please, make sure to call `optimizer.step()` before &quot;</span>
                              <span class="s2">&quot;`lr_scheduler.step()`. See more details at &quot;</span>
                              <span class="s2">&quot;https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

            <span class="c1"># Just check if there were two first lr_scheduler.step() calls before optimizer.step()</span>
            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">_step_count</span> <span class="o">&lt;</span> <span class="mi">1</span><span class="p">:</span>
                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;Detected call of `lr_scheduler.step()` before `optimizer.step()`. &quot;</span>
                              <span class="s2">&quot;In PyTorch 1.1.0 and later, you should call them in the opposite order: &quot;</span>
                              <span class="s2">&quot;`optimizer.step()` before `lr_scheduler.step()`.  Failure to do this &quot;</span>
                              <span class="s2">&quot;will result in PyTorch skipping the first value of the learning rate schedule. &quot;</span>
                              <span class="s2">&quot;See more details at &quot;</span>
                              <span class="s2">&quot;https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_step_count</span> <span class="o">+=</span> <span class="mi">1</span>

        <span class="k">class</span> <span class="nc">_enable_get_lr_call</span><span class="p">:</span>

            <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">o</span><span class="p">):</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">o</span> <span class="o">=</span> <span class="n">o</span>

            <span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">o</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span> <span class="o">=</span> <span class="kc">True</span>
                <span class="k">return</span> <span class="bp">self</span>

            <span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">type</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">traceback</span><span class="p">):</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">o</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span> <span class="o">=</span> <span class="kc">False</span>

        <span class="k">with</span> <span class="n">_enable_get_lr_call</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">epoch</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">+=</span> <span class="mi">1</span>
                <span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_lr</span><span class="p">()</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">EPOCH_DEPRECATION_WARNING</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">=</span> <span class="n">epoch</span>
                <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="s2">&quot;_get_closed_form_lr&quot;</span><span class="p">):</span>
                    <span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_closed_form_lr</span><span class="p">()</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">values</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_lr</span><span class="p">()</span>

        <span class="k">for</span> <span class="n">param_group</span><span class="p">,</span> <span class="n">lr</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">,</span> <span class="n">values</span><span class="p">):</span>
            <span class="n">param_group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">lr</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_last_lr</span> <span class="o">=</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span>


<div class="viewcode-block" id="LambdaLR"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.LambdaLR">[docs]</a><span class="k">class</span> <span class="nc">LambdaLR</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Sets the learning rate of each parameter group to the initial lr</span>
<span class="sd">    times a given function. When last_epoch=-1, sets initial lr as lr.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        lr_lambda (function or list): A function which computes a multiplicative</span>
<span class="sd">            factor given an integer parameter epoch, or a list of such</span>
<span class="sd">            functions, one for each group in optimizer.param_groups.</span>
<span class="sd">        last_epoch (int): The index of last epoch. Default: -1.</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; # Assuming optimizer has two groups.</span>
<span class="sd">        &gt;&gt;&gt; lambda1 = lambda epoch: epoch // 30</span>
<span class="sd">        &gt;&gt;&gt; lambda2 = lambda epoch: 0.95 ** epoch</span>
<span class="sd">        &gt;&gt;&gt; scheduler = LambdaLR(optimizer, lr_lambda=[lambda1, lambda2])</span>
<span class="sd">        &gt;&gt;&gt; for epoch in range(100):</span>
<span class="sd">        &gt;&gt;&gt;     train(...)</span>
<span class="sd">        &gt;&gt;&gt;     validate(...)</span>
<span class="sd">        &gt;&gt;&gt;     scheduler.step()</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">lr_lambda</span><span class="p">,</span> <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optimizer</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span> <span class="o">=</span> <span class="p">[</span><span class="n">lr_lambda</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected </span><span class="si">{}</span><span class="s2"> lr_lambdas, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                    <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">)))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">=</span> <span class="n">last_epoch</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">LambdaLR</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>

<div class="viewcode-block" id="LambdaLR.state_dict"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.LambdaLR.state_dict">[docs]</a>    <span class="k">def</span> <span class="nf">state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Returns the state of the scheduler as a :class:`dict`.</span>

<span class="sd">        It contains an entry for every variable in self.__dict__ which</span>
<span class="sd">        is not the optimizer.</span>
<span class="sd">        The learning rate lambda functions will only be saved if they are callable objects</span>
<span class="sd">        and not if they are functions or lambdas.</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">SAVE_STATE_WARNING</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>
        <span class="n">state_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="n">value</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;optimizer&#39;</span><span class="p">,</span> <span class="s1">&#39;lr_lambdas&#39;</span><span class="p">)}</span>
        <span class="n">state_dict</span><span class="p">[</span><span class="s1">&#39;lr_lambdas&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="kc">None</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span><span class="p">)</span>

        <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">fn</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span><span class="p">):</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">fn</span><span class="p">,</span> <span class="n">types</span><span class="o">.</span><span class="n">FunctionType</span><span class="p">):</span>
                <span class="n">state_dict</span><span class="p">[</span><span class="s1">&#39;lr_lambdas&#39;</span><span class="p">][</span><span class="n">idx</span><span class="p">]</span> <span class="o">=</span> <span class="n">fn</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

        <span class="k">return</span> <span class="n">state_dict</span></div>

<div class="viewcode-block" id="LambdaLR.load_state_dict"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.LambdaLR.load_state_dict">[docs]</a>    <span class="k">def</span> <span class="nf">load_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dict</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Loads the schedulers state.</span>

<span class="sd">        Arguments:</span>
<span class="sd">            state_dict (dict): scheduler state. Should be an object returned</span>
<span class="sd">                from a call to :meth:`state_dict`.</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">SAVE_STATE_WARNING</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>
        <span class="n">lr_lambdas</span> <span class="o">=</span> <span class="n">state_dict</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;lr_lambdas&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">state_dict</span><span class="p">)</span>
        <span class="c1"># Restore state_dict keys in order to prevent side effects</span>
        <span class="c1"># https://github.com/pytorch/pytorch/issues/32756</span>
        <span class="n">state_dict</span><span class="p">[</span><span class="s1">&#39;lr_lambdas&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">lr_lambdas</span>

        <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">fn</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">lr_lambdas</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span></div>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">)</span>

        <span class="k">return</span> <span class="p">[</span><span class="n">base_lr</span> <span class="o">*</span> <span class="n">lmbda</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span><span class="p">)</span>
                <span class="k">for</span> <span class="n">lmbda</span><span class="p">,</span> <span class="n">base_lr</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">)]</span></div>


<div class="viewcode-block" id="MultiplicativeLR"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.MultiplicativeLR">[docs]</a><span class="k">class</span> <span class="nc">MultiplicativeLR</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Multiply the learning rate of each parameter group by the factor given</span>
<span class="sd">    in the specified function. When last_epoch=-1, sets initial lr as lr.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        lr_lambda (function or list): A function which computes a multiplicative</span>
<span class="sd">            factor given an integer parameter epoch, or a list of such</span>
<span class="sd">            functions, one for each group in optimizer.param_groups.</span>
<span class="sd">        last_epoch (int): The index of last epoch. Default: -1.</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; lmbda = lambda epoch: 0.95</span>
<span class="sd">        &gt;&gt;&gt; scheduler = MultiplicativeLR(optimizer, lr_lambda=lmbda)</span>
<span class="sd">        &gt;&gt;&gt; for epoch in range(100):</span>
<span class="sd">        &gt;&gt;&gt;     train(...)</span>
<span class="sd">        &gt;&gt;&gt;     validate(...)</span>
<span class="sd">        &gt;&gt;&gt;     scheduler.step()</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">lr_lambda</span><span class="p">,</span> <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optimizer</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="ow">and</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span> <span class="o">=</span> <span class="p">[</span><span class="n">lr_lambda</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected </span><span class="si">{}</span><span class="s2"> lr_lambdas, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                    <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">)))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">lr_lambda</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">=</span> <span class="n">last_epoch</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">MultiplicativeLR</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>

<div class="viewcode-block" id="MultiplicativeLR.state_dict"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.MultiplicativeLR.state_dict">[docs]</a>    <span class="k">def</span> <span class="nf">state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Returns the state of the scheduler as a :class:`dict`.</span>

<span class="sd">        It contains an entry for every variable in self.__dict__ which</span>
<span class="sd">        is not the optimizer.</span>
<span class="sd">        The learning rate lambda functions will only be saved if they are callable objects</span>
<span class="sd">        and not if they are functions or lambdas.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">state_dict</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="n">value</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">key</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">(</span><span class="s1">&#39;optimizer&#39;</span><span class="p">,</span> <span class="s1">&#39;lr_lambdas&#39;</span><span class="p">)}</span>
        <span class="n">state_dict</span><span class="p">[</span><span class="s1">&#39;lr_lambdas&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">[</span><span class="kc">None</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span><span class="p">)</span>

        <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">fn</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span><span class="p">):</span>
            <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">fn</span><span class="p">,</span> <span class="n">types</span><span class="o">.</span><span class="n">FunctionType</span><span class="p">):</span>
                <span class="n">state_dict</span><span class="p">[</span><span class="s1">&#39;lr_lambdas&#39;</span><span class="p">][</span><span class="n">idx</span><span class="p">]</span> <span class="o">=</span> <span class="n">fn</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">copy</span><span class="p">()</span>

        <span class="k">return</span> <span class="n">state_dict</span></div>

<div class="viewcode-block" id="MultiplicativeLR.load_state_dict"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.MultiplicativeLR.load_state_dict">[docs]</a>    <span class="k">def</span> <span class="nf">load_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dict</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Loads the schedulers state.</span>

<span class="sd">        Arguments:</span>
<span class="sd">            state_dict (dict): scheduler state. Should be an object returned</span>
<span class="sd">                from a call to :meth:`state_dict`.</span>
<span class="sd">        &quot;&quot;&quot;</span>
        <span class="n">lr_lambdas</span> <span class="o">=</span> <span class="n">state_dict</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="s1">&#39;lr_lambdas&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">state_dict</span><span class="p">)</span>
        <span class="c1"># Restore state_dict keys in order to prevent side effects</span>
        <span class="c1"># https://github.com/pytorch/pytorch/issues/32756</span>
        <span class="n">state_dict</span><span class="p">[</span><span class="s1">&#39;lr_lambdas&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">lr_lambdas</span>

        <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">fn</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="n">lr_lambdas</span><span class="p">):</span>
            <span class="k">if</span> <span class="n">fn</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">fn</span><span class="p">)</span></div>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">return</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">*</span> <span class="n">lmbda</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span><span class="p">)</span>
                    <span class="k">for</span> <span class="n">lmbda</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">lr_lambdas</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">)]</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">return</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">)</span></div>


<div class="viewcode-block" id="StepLR"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.StepLR">[docs]</a><span class="k">class</span> <span class="nc">StepLR</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Decays the learning rate of each parameter group by gamma every</span>
<span class="sd">    step_size epochs. Notice that such decay can happen simultaneously with</span>
<span class="sd">    other changes to the learning rate from outside this scheduler. When</span>
<span class="sd">    last_epoch=-1, sets initial lr as lr.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        step_size (int): Period of learning rate decay.</span>
<span class="sd">        gamma (float): Multiplicative factor of learning rate decay.</span>
<span class="sd">            Default: 0.1.</span>
<span class="sd">        last_epoch (int): The index of last epoch. Default: -1.</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; # Assuming optimizer uses lr = 0.05 for all groups</span>
<span class="sd">        &gt;&gt;&gt; # lr = 0.05     if epoch &lt; 30</span>
<span class="sd">        &gt;&gt;&gt; # lr = 0.005    if 30 &lt;= epoch &lt; 60</span>
<span class="sd">        &gt;&gt;&gt; # lr = 0.0005   if 60 &lt;= epoch &lt; 90</span>
<span class="sd">        &gt;&gt;&gt; # ...</span>
<span class="sd">        &gt;&gt;&gt; scheduler = StepLR(optimizer, step_size=30, gamma=0.1)</span>
<span class="sd">        &gt;&gt;&gt; for epoch in range(100):</span>
<span class="sd">        &gt;&gt;&gt;     train(...)</span>
<span class="sd">        &gt;&gt;&gt;     validate(...)</span>
<span class="sd">        &gt;&gt;&gt;     scheduler.step()</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">step_size</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">step_size</span> <span class="o">=</span> <span class="n">step_size</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">StepLR</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

        <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">==</span> <span class="mi">0</span><span class="p">)</span> <span class="ow">or</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size</span> <span class="o">!=</span> <span class="mi">0</span><span class="p">):</span>
            <span class="k">return</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span>
                <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">_get_closed_form_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">base_lr</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">**</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">//</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size</span><span class="p">)</span>
                <span class="k">for</span> <span class="n">base_lr</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">]</span></div>


<div class="viewcode-block" id="MultiStepLR"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.MultiStepLR">[docs]</a><span class="k">class</span> <span class="nc">MultiStepLR</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Decays the learning rate of each parameter group by gamma once the</span>
<span class="sd">    number of epoch reaches one of the milestones. Notice that such decay can</span>
<span class="sd">    happen simultaneously with other changes to the learning rate from outside</span>
<span class="sd">    this scheduler. When last_epoch=-1, sets initial lr as lr.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        milestones (list): List of epoch indices. Must be increasing.</span>
<span class="sd">        gamma (float): Multiplicative factor of learning rate decay.</span>
<span class="sd">            Default: 0.1.</span>
<span class="sd">        last_epoch (int): The index of last epoch. Default: -1.</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; # Assuming optimizer uses lr = 0.05 for all groups</span>
<span class="sd">        &gt;&gt;&gt; # lr = 0.05     if epoch &lt; 30</span>
<span class="sd">        &gt;&gt;&gt; # lr = 0.005    if 30 &lt;= epoch &lt; 80</span>
<span class="sd">        &gt;&gt;&gt; # lr = 0.0005   if epoch &gt;= 80</span>
<span class="sd">        &gt;&gt;&gt; scheduler = MultiStepLR(optimizer, milestones=[30,80], gamma=0.1)</span>
<span class="sd">        &gt;&gt;&gt; for epoch in range(100):</span>
<span class="sd">        &gt;&gt;&gt;     train(...)</span>
<span class="sd">        &gt;&gt;&gt;     validate(...)</span>
<span class="sd">        &gt;&gt;&gt;     scheduler.step()</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">milestones</span><span class="p">,</span> <span class="n">gamma</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">milestones</span> <span class="o">=</span> <span class="n">Counter</span><span class="p">(</span><span class="n">milestones</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">MultiStepLR</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">milestones</span><span class="p">:</span>
            <span class="k">return</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">milestones</span><span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span><span class="p">]</span>
                <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">_get_closed_form_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="n">milestones</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">sorted</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">milestones</span><span class="o">.</span><span class="n">elements</span><span class="p">()))</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">base_lr</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">**</span> <span class="n">bisect_right</span><span class="p">(</span><span class="n">milestones</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span><span class="p">)</span>
                <span class="k">for</span> <span class="n">base_lr</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">]</span></div>


<div class="viewcode-block" id="ExponentialLR"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.ExponentialLR">[docs]</a><span class="k">class</span> <span class="nc">ExponentialLR</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Decays the learning rate of each parameter group by gamma every epoch.</span>
<span class="sd">    When last_epoch=-1, sets initial lr as lr.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        gamma (float): Multiplicative factor of learning rate decay.</span>
<span class="sd">        last_epoch (int): The index of last epoch. Default: -1.</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">gamma</span><span class="p">,</span> <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">ExponentialLR</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span>
                <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">_get_closed_form_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="p">[</span><span class="n">base_lr</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">**</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span>
                <span class="k">for</span> <span class="n">base_lr</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">]</span></div>


<div class="viewcode-block" id="CosineAnnealingLR"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.CosineAnnealingLR">[docs]</a><span class="k">class</span> <span class="nc">CosineAnnealingLR</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Set the learning rate of each parameter group using a cosine annealing</span>
<span class="sd">    schedule, where :math:`\eta_{max}` is set to the initial lr and</span>
<span class="sd">    :math:`T_{cur}` is the number of epochs since the last restart in SGDR:</span>

<span class="sd">    .. math::</span>
<span class="sd">        \begin{aligned}</span>
<span class="sd">            \eta_t &amp; = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1</span>
<span class="sd">            + \cos\left(\frac{T_{cur}}{T_{max}}\pi\right)\right),</span>
<span class="sd">            &amp; T_{cur} \neq (2k+1)T_{max}; \\</span>
<span class="sd">            \eta_{t+1} &amp; = \eta_{t} + \frac{1}{2}(\eta_{max} - \eta_{min})</span>
<span class="sd">            \left(1 - \cos\left(\frac{1}{T_{max}}\pi\right)\right),</span>
<span class="sd">            &amp; T_{cur} = (2k+1)T_{max}.</span>
<span class="sd">        \end{aligned}</span>

<span class="sd">    When last_epoch=-1, sets initial lr as lr. Notice that because the schedule</span>
<span class="sd">    is defined recursively, the learning rate can be simultaneously modified</span>
<span class="sd">    outside this scheduler by other operators. If the learning rate is set</span>
<span class="sd">    solely by this scheduler, the learning rate at each step becomes:</span>

<span class="sd">    .. math::</span>
<span class="sd">        \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1 +</span>
<span class="sd">        \cos\left(\frac{T_{cur}}{T_{max}}\pi\right)\right)</span>

<span class="sd">    It has been proposed in</span>
<span class="sd">    `SGDR: Stochastic Gradient Descent with Warm Restarts`_. Note that this only</span>
<span class="sd">    implements the cosine annealing part of SGDR, and not the restarts.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        T_max (int): Maximum number of iterations.</span>
<span class="sd">        eta_min (float): Minimum learning rate. Default: 0.</span>
<span class="sd">        last_epoch (int): The index of last epoch. Default: -1.</span>

<span class="sd">    .. _SGDR\: Stochastic Gradient Descent with Warm Restarts:</span>
<span class="sd">        https://arxiv.org/abs/1608.03983</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">T_max</span><span class="p">,</span> <span class="n">eta_min</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">T_max</span> <span class="o">=</span> <span class="n">T_max</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span> <span class="o">=</span> <span class="n">eta_min</span>
        <span class="nb">super</span><span class="p">(</span><span class="n">CosineAnnealingLR</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span>
        <span class="k">elif</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">-</span> <span class="mi">1</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_max</span><span class="p">)</span> <span class="o">%</span> <span class="p">(</span><span class="mi">2</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_max</span><span class="p">)</span> <span class="o">==</span> <span class="mi">0</span><span class="p">:</span>
            <span class="k">return</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">+</span> <span class="p">(</span><span class="n">base_lr</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span><span class="p">)</span> <span class="o">*</span>
                    <span class="p">(</span><span class="mi">1</span> <span class="o">-</span> <span class="n">math</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">pi</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_max</span><span class="p">))</span> <span class="o">/</span> <span class="mi">2</span>
                    <span class="k">for</span> <span class="n">base_lr</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span>
                    <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">)]</span>
        <span class="k">return</span> <span class="p">[(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">math</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">pi</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_max</span><span class="p">))</span> <span class="o">/</span>
                <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">math</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">pi</span> <span class="o">*</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_max</span><span class="p">))</span> <span class="o">*</span>
                <span class="p">(</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span><span class="p">)</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span>
                <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">_get_closed_form_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span> <span class="o">+</span> <span class="p">(</span><span class="n">base_lr</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span><span class="p">)</span> <span class="o">*</span>
                <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">math</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">pi</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_max</span><span class="p">))</span> <span class="o">/</span> <span class="mi">2</span>
                <span class="k">for</span> <span class="n">base_lr</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">]</span></div>


<div class="viewcode-block" id="ReduceLROnPlateau"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.ReduceLROnPlateau">[docs]</a><span class="k">class</span> <span class="nc">ReduceLROnPlateau</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="sd">&quot;&quot;&quot;Reduce learning rate when a metric has stopped improving.</span>
<span class="sd">    Models often benefit from reducing the learning rate by a factor</span>
<span class="sd">    of 2-10 once learning stagnates. This scheduler reads a metrics</span>
<span class="sd">    quantity and if no improvement is seen for a &#39;patience&#39; number</span>
<span class="sd">    of epochs, the learning rate is reduced.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        mode (str): One of `min`, `max`. In `min` mode, lr will</span>
<span class="sd">            be reduced when the quantity monitored has stopped</span>
<span class="sd">            decreasing; in `max` mode it will be reduced when the</span>
<span class="sd">            quantity monitored has stopped increasing. Default: &#39;min&#39;.</span>
<span class="sd">        factor (float): Factor by which the learning rate will be</span>
<span class="sd">            reduced. new_lr = lr * factor. Default: 0.1.</span>
<span class="sd">        patience (int): Number of epochs with no improvement after</span>
<span class="sd">            which learning rate will be reduced. For example, if</span>
<span class="sd">            `patience = 2`, then we will ignore the first 2 epochs</span>
<span class="sd">            with no improvement, and will only decrease the LR after the</span>
<span class="sd">            3rd epoch if the loss still hasn&#39;t improved then.</span>
<span class="sd">            Default: 10.</span>
<span class="sd">        verbose (bool): If ``True``, prints a message to stdout for</span>
<span class="sd">            each update. Default: ``False``.</span>
<span class="sd">        threshold (float): Threshold for measuring the new optimum,</span>
<span class="sd">            to only focus on significant changes. Default: 1e-4.</span>
<span class="sd">        threshold_mode (str): One of `rel`, `abs`. In `rel` mode,</span>
<span class="sd">            dynamic_threshold = best * ( 1 + threshold ) in &#39;max&#39;</span>
<span class="sd">            mode or best * ( 1 - threshold ) in `min` mode.</span>
<span class="sd">            In `abs` mode, dynamic_threshold = best + threshold in</span>
<span class="sd">            `max` mode or best - threshold in `min` mode. Default: &#39;rel&#39;.</span>
<span class="sd">        cooldown (int): Number of epochs to wait before resuming</span>
<span class="sd">            normal operation after lr has been reduced. Default: 0.</span>
<span class="sd">        min_lr (float or list): A scalar or a list of scalars. A</span>
<span class="sd">            lower bound on the learning rate of all param groups</span>
<span class="sd">            or each group respectively. Default: 0.</span>
<span class="sd">        eps (float): Minimal decay applied to lr. If the difference</span>
<span class="sd">            between new and old lr is smaller than eps, the update is</span>
<span class="sd">            ignored. Default: 1e-8.</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)</span>
<span class="sd">        &gt;&gt;&gt; scheduler = ReduceLROnPlateau(optimizer, &#39;min&#39;)</span>
<span class="sd">        &gt;&gt;&gt; for epoch in range(10):</span>
<span class="sd">        &gt;&gt;&gt;     train(...)</span>
<span class="sd">        &gt;&gt;&gt;     val_loss = validate(...)</span>
<span class="sd">        &gt;&gt;&gt;     # Note that step should be called after validate()</span>
<span class="sd">        &gt;&gt;&gt;     scheduler.step(val_loss)</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;min&#39;</span><span class="p">,</span> <span class="n">factor</span><span class="o">=</span><span class="mf">0.1</span><span class="p">,</span> <span class="n">patience</span><span class="o">=</span><span class="mi">10</span><span class="p">,</span>
                 <span class="n">verbose</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">threshold</span><span class="o">=</span><span class="mf">1e-4</span><span class="p">,</span> <span class="n">threshold_mode</span><span class="o">=</span><span class="s1">&#39;rel&#39;</span><span class="p">,</span>
                 <span class="n">cooldown</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">min_lr</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">eps</span><span class="o">=</span><span class="mf">1e-8</span><span class="p">):</span>

        <span class="k">if</span> <span class="n">factor</span> <span class="o">&gt;=</span> <span class="mf">1.0</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;Factor should be &lt; 1.0.&#39;</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">factor</span> <span class="o">=</span> <span class="n">factor</span>

        <span class="c1"># Attach optimizer</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">Optimizer</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> is not an Optimizer&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="nb">type</span><span class="p">(</span><span class="n">optimizer</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optimizer</span>

        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">min_lr</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="ow">or</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">min_lr</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">):</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">min_lr</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;expected </span><span class="si">{}</span><span class="s2"> min_lrs, got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                    <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">),</span> <span class="nb">len</span><span class="p">(</span><span class="n">min_lr</span><span class="p">)))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">min_lrs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">min_lr</span><span class="p">)</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">min_lrs</span> <span class="o">=</span> <span class="p">[</span><span class="n">min_lr</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">patience</span> <span class="o">=</span> <span class="n">patience</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span> <span class="o">=</span> <span class="n">verbose</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cooldown</span> <span class="o">=</span> <span class="n">cooldown</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cooldown_counter</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="n">mode</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">threshold</span> <span class="o">=</span> <span class="n">threshold</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">threshold_mode</span> <span class="o">=</span> <span class="n">threshold_mode</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_bad_epochs</span> <span class="o">=</span> <span class="kc">None</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">mode_worse</span> <span class="o">=</span> <span class="kc">None</span>  <span class="c1"># the worse value for the chosen mode</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">eps</span> <span class="o">=</span> <span class="n">eps</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_init_is_better</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="n">mode</span><span class="p">,</span> <span class="n">threshold</span><span class="o">=</span><span class="n">threshold</span><span class="p">,</span>
                             <span class="n">threshold_mode</span><span class="o">=</span><span class="n">threshold_mode</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_reset</span><span class="p">()</span>

    <span class="k">def</span> <span class="nf">_reset</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Resets num_bad_epochs counter and cooldown counter.&quot;&quot;&quot;</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode_worse</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cooldown_counter</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">num_bad_epochs</span> <span class="o">=</span> <span class="mi">0</span>

    <span class="k">def</span> <span class="nf">step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">metrics</span><span class="p">,</span> <span class="n">epoch</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="c1"># convert `metrics` to float, in case it&#39;s a zero-dim Tensor</span>
        <span class="n">current</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">metrics</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">epoch</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">epoch</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">+</span> <span class="mi">1</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="n">EPOCH_DEPRECATION_WARNING</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">=</span> <span class="n">epoch</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">is_better</span><span class="p">(</span><span class="n">current</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">best</span><span class="p">):</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">best</span> <span class="o">=</span> <span class="n">current</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">num_bad_epochs</span> <span class="o">=</span> <span class="mi">0</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">num_bad_epochs</span> <span class="o">+=</span> <span class="mi">1</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">in_cooldown</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cooldown_counter</span> <span class="o">-=</span> <span class="mi">1</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">num_bad_epochs</span> <span class="o">=</span> <span class="mi">0</span>  <span class="c1"># ignore any bad epochs in cooldown</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">num_bad_epochs</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">patience</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">_reduce_lr</span><span class="p">(</span><span class="n">epoch</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">cooldown_counter</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cooldown</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">num_bad_epochs</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_last_lr</span> <span class="o">=</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span>

    <span class="k">def</span> <span class="nf">_reduce_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">epoch</span><span class="p">):</span>
        <span class="k">for</span> <span class="n">i</span><span class="p">,</span> <span class="n">param_group</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
            <span class="n">old_lr</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">param_group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">])</span>
            <span class="n">new_lr</span> <span class="o">=</span> <span class="nb">max</span><span class="p">(</span><span class="n">old_lr</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">factor</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">min_lrs</span><span class="p">[</span><span class="n">i</span><span class="p">])</span>
            <span class="k">if</span> <span class="n">old_lr</span> <span class="o">-</span> <span class="n">new_lr</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">eps</span><span class="p">:</span>
                <span class="n">param_group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">new_lr</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">verbose</span><span class="p">:</span>
                    <span class="nb">print</span><span class="p">(</span><span class="s1">&#39;Epoch </span><span class="si">{:5d}</span><span class="s1">: reducing learning rate&#39;</span>
                          <span class="s1">&#39; of group </span><span class="si">{}</span><span class="s1"> to </span><span class="si">{:.4e}</span><span class="s1">.&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">epoch</span><span class="p">,</span> <span class="n">i</span><span class="p">,</span> <span class="n">new_lr</span><span class="p">))</span>

    <span class="nd">@property</span>
    <span class="k">def</span> <span class="nf">in_cooldown</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">cooldown_counter</span> <span class="o">&gt;</span> <span class="mi">0</span>

    <span class="k">def</span> <span class="nf">is_better</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">a</span><span class="p">,</span> <span class="n">best</span><span class="p">):</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;min&#39;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">threshold_mode</span> <span class="o">==</span> <span class="s1">&#39;rel&#39;</span><span class="p">:</span>
            <span class="n">rel_epsilon</span> <span class="o">=</span> <span class="mf">1.</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">threshold</span>
            <span class="k">return</span> <span class="n">a</span> <span class="o">&lt;</span> <span class="n">best</span> <span class="o">*</span> <span class="n">rel_epsilon</span>

        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;min&#39;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">threshold_mode</span> <span class="o">==</span> <span class="s1">&#39;abs&#39;</span><span class="p">:</span>
            <span class="k">return</span> <span class="n">a</span> <span class="o">&lt;</span> <span class="n">best</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">threshold</span>

        <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;max&#39;</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">threshold_mode</span> <span class="o">==</span> <span class="s1">&#39;rel&#39;</span><span class="p">:</span>
            <span class="n">rel_epsilon</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">threshold</span> <span class="o">+</span> <span class="mf">1.</span>
            <span class="k">return</span> <span class="n">a</span> <span class="o">&gt;</span> <span class="n">best</span> <span class="o">*</span> <span class="n">rel_epsilon</span>

        <span class="k">else</span><span class="p">:</span>  <span class="c1"># mode == &#39;max&#39; and epsilon_mode == &#39;abs&#39;:</span>
            <span class="k">return</span> <span class="n">a</span> <span class="o">&gt;</span> <span class="n">best</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">threshold</span>

    <span class="k">def</span> <span class="nf">_init_is_better</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">mode</span><span class="p">,</span> <span class="n">threshold</span><span class="p">,</span> <span class="n">threshold_mode</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">mode</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">{</span><span class="s1">&#39;min&#39;</span><span class="p">,</span> <span class="s1">&#39;max&#39;</span><span class="p">}:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;mode &#39;</span> <span class="o">+</span> <span class="n">mode</span> <span class="o">+</span> <span class="s1">&#39; is unknown!&#39;</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">threshold_mode</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">{</span><span class="s1">&#39;rel&#39;</span><span class="p">,</span> <span class="s1">&#39;abs&#39;</span><span class="p">}:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;threshold mode &#39;</span> <span class="o">+</span> <span class="n">threshold_mode</span> <span class="o">+</span> <span class="s1">&#39; is unknown!&#39;</span><span class="p">)</span>

        <span class="k">if</span> <span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;min&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">mode_worse</span> <span class="o">=</span> <span class="n">inf</span>
        <span class="k">else</span><span class="p">:</span>  <span class="c1"># mode == &#39;max&#39;:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">mode_worse</span> <span class="o">=</span> <span class="o">-</span><span class="n">inf</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="n">mode</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">threshold</span> <span class="o">=</span> <span class="n">threshold</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">threshold_mode</span> <span class="o">=</span> <span class="n">threshold_mode</span>

    <span class="k">def</span> <span class="nf">state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">return</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="n">value</span> <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">value</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">items</span><span class="p">()</span> <span class="k">if</span> <span class="n">key</span> <span class="o">!=</span> <span class="s1">&#39;optimizer&#39;</span><span class="p">}</span>

    <span class="k">def</span> <span class="nf">load_state_dict</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">state_dict</span><span class="p">):</span>
        <span class="bp">self</span><span class="o">.</span><span class="vm">__dict__</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">state_dict</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">_init_is_better</span><span class="p">(</span><span class="n">mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">mode</span><span class="p">,</span> <span class="n">threshold</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">threshold</span><span class="p">,</span> <span class="n">threshold_mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">threshold_mode</span><span class="p">)</span></div>


<div class="viewcode-block" id="CyclicLR"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.CyclicLR">[docs]</a><span class="k">class</span> <span class="nc">CyclicLR</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Sets the learning rate of each parameter group according to</span>
<span class="sd">    cyclical learning rate policy (CLR). The policy cycles the learning</span>
<span class="sd">    rate between two boundaries with a constant frequency, as detailed in</span>
<span class="sd">    the paper `Cyclical Learning Rates for Training Neural Networks`_.</span>
<span class="sd">    The distance between the two boundaries can be scaled on a per-iteration</span>
<span class="sd">    or per-cycle basis.</span>

<span class="sd">    Cyclical learning rate policy changes the learning rate after every batch.</span>
<span class="sd">    `step` should be called after a batch has been used for training.</span>

<span class="sd">    This class has three built-in policies, as put forth in the paper:</span>

<span class="sd">    * &quot;triangular&quot;: A basic triangular cycle without amplitude scaling.</span>
<span class="sd">    * &quot;triangular2&quot;: A basic triangular cycle that scales initial amplitude by half each cycle.</span>
<span class="sd">    * &quot;exp_range&quot;: A cycle that scales initial amplitude by :math:`\text{gamma}^{\text{cycle iterations}}`</span>
<span class="sd">      at each cycle iteration.</span>

<span class="sd">    This implementation was adapted from the github repo: `bckenstler/CLR`_</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        base_lr (float or list): Initial learning rate which is the</span>
<span class="sd">            lower boundary in the cycle for each parameter group.</span>
<span class="sd">        max_lr (float or list): Upper learning rate boundaries in the cycle</span>
<span class="sd">            for each parameter group. Functionally,</span>
<span class="sd">            it defines the cycle amplitude (max_lr - base_lr).</span>
<span class="sd">            The lr at any cycle is the sum of base_lr</span>
<span class="sd">            and some scaling of the amplitude; therefore</span>
<span class="sd">            max_lr may not actually be reached depending on</span>
<span class="sd">            scaling function.</span>
<span class="sd">        step_size_up (int): Number of training iterations in the</span>
<span class="sd">            increasing half of a cycle. Default: 2000</span>
<span class="sd">        step_size_down (int): Number of training iterations in the</span>
<span class="sd">            decreasing half of a cycle. If step_size_down is None,</span>
<span class="sd">            it is set to step_size_up. Default: None</span>
<span class="sd">        mode (str): One of {triangular, triangular2, exp_range}.</span>
<span class="sd">            Values correspond to policies detailed above.</span>
<span class="sd">            If scale_fn is not None, this argument is ignored.</span>
<span class="sd">            Default: &#39;triangular&#39;</span>
<span class="sd">        gamma (float): Constant in &#39;exp_range&#39; scaling function:</span>
<span class="sd">            gamma**(cycle iterations)</span>
<span class="sd">            Default: 1.0</span>
<span class="sd">        scale_fn (function): Custom scaling policy defined by a single</span>
<span class="sd">            argument lambda function, where</span>
<span class="sd">            0 &lt;= scale_fn(x) &lt;= 1 for all x &gt;= 0.</span>
<span class="sd">            If specified, then &#39;mode&#39; is ignored.</span>
<span class="sd">            Default: None</span>
<span class="sd">        scale_mode (str): {&#39;cycle&#39;, &#39;iterations&#39;}.</span>
<span class="sd">            Defines whether scale_fn is evaluated on</span>
<span class="sd">            cycle number or cycle iterations (training</span>
<span class="sd">            iterations since start of cycle).</span>
<span class="sd">            Default: &#39;cycle&#39;</span>
<span class="sd">        cycle_momentum (bool): If ``True``, momentum is cycled inversely</span>
<span class="sd">            to learning rate between &#39;base_momentum&#39; and &#39;max_momentum&#39;.</span>
<span class="sd">            Default: True</span>
<span class="sd">        base_momentum (float or list): Lower momentum boundaries in the cycle</span>
<span class="sd">            for each parameter group. Note that momentum is cycled inversely</span>
<span class="sd">            to learning rate; at the peak of a cycle, momentum is</span>
<span class="sd">            &#39;base_momentum&#39; and learning rate is &#39;max_lr&#39;.</span>
<span class="sd">            Default: 0.8</span>
<span class="sd">        max_momentum (float or list): Upper momentum boundaries in the cycle</span>
<span class="sd">            for each parameter group. Functionally,</span>
<span class="sd">            it defines the cycle amplitude (max_momentum - base_momentum).</span>
<span class="sd">            The momentum at any cycle is the difference of max_momentum</span>
<span class="sd">            and some scaling of the amplitude; therefore</span>
<span class="sd">            base_momentum may not actually be reached depending on</span>
<span class="sd">            scaling function. Note that momentum is cycled inversely</span>
<span class="sd">            to learning rate; at the start of a cycle, momentum is &#39;max_momentum&#39;</span>
<span class="sd">            and learning rate is &#39;base_lr&#39;</span>
<span class="sd">            Default: 0.9</span>
<span class="sd">        last_epoch (int): The index of the last batch. This parameter is used when</span>
<span class="sd">            resuming a training job. Since `step()` should be invoked after each</span>
<span class="sd">            batch instead of after each epoch, this number represents the total</span>
<span class="sd">            number of *batches* computed, not the total number of epochs computed.</span>
<span class="sd">            When last_epoch=-1, the schedule is started from the beginning.</span>
<span class="sd">            Default: -1</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)</span>
<span class="sd">        &gt;&gt;&gt; scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.01, max_lr=0.1)</span>
<span class="sd">        &gt;&gt;&gt; data_loader = torch.utils.data.DataLoader(...)</span>
<span class="sd">        &gt;&gt;&gt; for epoch in range(10):</span>
<span class="sd">        &gt;&gt;&gt;     for batch in data_loader:</span>
<span class="sd">        &gt;&gt;&gt;         train_batch(...)</span>
<span class="sd">        &gt;&gt;&gt;         scheduler.step()</span>


<span class="sd">    .. _Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186</span>
<span class="sd">    .. _bckenstler/CLR: https://github.com/bckenstler/CLR</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">optimizer</span><span class="p">,</span>
                 <span class="n">base_lr</span><span class="p">,</span>
                 <span class="n">max_lr</span><span class="p">,</span>
                 <span class="n">step_size_up</span><span class="o">=</span><span class="mi">2000</span><span class="p">,</span>
                 <span class="n">step_size_down</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">mode</span><span class="o">=</span><span class="s1">&#39;triangular&#39;</span><span class="p">,</span>
                 <span class="n">gamma</span><span class="o">=</span><span class="mf">1.</span><span class="p">,</span>
                 <span class="n">scale_fn</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">scale_mode</span><span class="o">=</span><span class="s1">&#39;cycle&#39;</span><span class="p">,</span>
                 <span class="n">cycle_momentum</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">base_momentum</span><span class="o">=</span><span class="mf">0.8</span><span class="p">,</span>
                 <span class="n">max_momentum</span><span class="o">=</span><span class="mf">0.9</span><span class="p">,</span>
                 <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>

        <span class="c1"># Attach optimizer</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">Optimizer</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> is not an Optimizer&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="nb">type</span><span class="p">(</span><span class="n">optimizer</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optimizer</span>

        <span class="n">base_lrs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_format_param</span><span class="p">(</span><span class="s1">&#39;base_lr&#39;</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">base_lr</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">last_epoch</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">lr</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">base_lrs</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                <span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">lr</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">max_lrs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_format_param</span><span class="p">(</span><span class="s1">&#39;max_lr&#39;</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">max_lr</span><span class="p">)</span>

        <span class="n">step_size_up</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">step_size_up</span><span class="p">)</span>
        <span class="n">step_size_down</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">step_size_down</span><span class="p">)</span> <span class="k">if</span> <span class="n">step_size_down</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">step_size_up</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">total_size</span> <span class="o">=</span> <span class="n">step_size_up</span> <span class="o">+</span> <span class="n">step_size_down</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">step_ratio</span> <span class="o">=</span> <span class="n">step_size_up</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">total_size</span>

        <span class="k">if</span> <span class="n">mode</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;triangular&#39;</span><span class="p">,</span> <span class="s1">&#39;triangular2&#39;</span><span class="p">,</span> <span class="s1">&#39;exp_range&#39;</span><span class="p">]</span> \
                <span class="ow">and</span> <span class="n">scale_fn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;mode is invalid and scale_fn is None&#39;</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">=</span> <span class="n">mode</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span> <span class="o">=</span> <span class="n">gamma</span>

        <span class="k">if</span> <span class="n">scale_fn</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;triangular&#39;</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">scale_fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_triangular_scale_fn</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">scale_mode</span> <span class="o">=</span> <span class="s1">&#39;cycle&#39;</span>
            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;triangular2&#39;</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">scale_fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_triangular2_scale_fn</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">scale_mode</span> <span class="o">=</span> <span class="s1">&#39;cycle&#39;</span>
            <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">mode</span> <span class="o">==</span> <span class="s1">&#39;exp_range&#39;</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">scale_fn</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_exp_range_scale_fn</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">scale_mode</span> <span class="o">=</span> <span class="s1">&#39;iterations&#39;</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">scale_fn</span> <span class="o">=</span> <span class="n">scale_fn</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">scale_mode</span> <span class="o">=</span> <span class="n">scale_mode</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">cycle_momentum</span> <span class="o">=</span> <span class="n">cycle_momentum</span>
        <span class="k">if</span> <span class="n">cycle_momentum</span><span class="p">:</span>
            <span class="k">if</span> <span class="s1">&#39;momentum&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">defaults</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;optimizer must support momentum with `cycle_momentum` option enabled&#39;</span><span class="p">)</span>

            <span class="n">base_momentums</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_format_param</span><span class="p">(</span><span class="s1">&#39;base_momentum&#39;</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">base_momentum</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">last_epoch</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">momentum</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">base_momentums</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                    <span class="n">group</span><span class="p">[</span><span class="s1">&#39;momentum&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">momentum</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">base_momentums</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">map</span><span class="p">(</span><span class="k">lambda</span> <span class="n">group</span><span class="p">:</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;momentum&#39;</span><span class="p">],</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">max_momentums</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_format_param</span><span class="p">(</span><span class="s1">&#39;max_momentum&#39;</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">max_momentum</span><span class="p">)</span>

        <span class="nb">super</span><span class="p">(</span><span class="n">CyclicLR</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span> <span class="o">=</span> <span class="n">base_lrs</span>

    <span class="k">def</span> <span class="nf">_format_param</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">param</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Return correctly formatted lr/momentum for each param group.&quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">param</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">param</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;expected </span><span class="si">{}</span><span class="s2"> values for </span><span class="si">{}</span><span class="s2">, got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                    <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">),</span> <span class="n">name</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">param</span><span class="p">)))</span>
            <span class="k">return</span> <span class="n">param</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">return</span> <span class="p">[</span><span class="n">param</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_triangular_scale_fn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="mf">1.</span>

    <span class="k">def</span> <span class="nf">_triangular2_scale_fn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="mi">1</span> <span class="o">/</span> <span class="p">(</span><span class="mf">2.</span> <span class="o">**</span> <span class="p">(</span><span class="n">x</span> <span class="o">-</span> <span class="mi">1</span><span class="p">))</span>

    <span class="k">def</span> <span class="nf">_exp_range_scale_fn</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">x</span><span class="p">):</span>
        <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">gamma</span><span class="o">**</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>

<div class="viewcode-block" id="CyclicLR.get_lr"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.CyclicLR.get_lr">[docs]</a>    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Calculates the learning rate at batch index. This function treats</span>
<span class="sd">        `self.last_epoch` as the last batch index.</span>

<span class="sd">        If `self.cycle_momentum` is ``True``, this function has a side effect of</span>
<span class="sd">        updating the optimizer&#39;s momentum.</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

        <span class="n">cycle</span> <span class="o">=</span> <span class="n">math</span><span class="o">.</span><span class="n">floor</span><span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">total_size</span><span class="p">)</span>
        <span class="n">x</span> <span class="o">=</span> <span class="mf">1.</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">total_size</span> <span class="o">-</span> <span class="n">cycle</span>
        <span class="k">if</span> <span class="n">x</span> <span class="o">&lt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_ratio</span><span class="p">:</span>
            <span class="n">scale_factor</span> <span class="o">=</span> <span class="n">x</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_ratio</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="n">scale_factor</span> <span class="o">=</span> <span class="p">(</span><span class="n">x</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">step_ratio</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>

        <span class="n">lrs</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="k">for</span> <span class="n">base_lr</span><span class="p">,</span> <span class="n">max_lr</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_lrs</span><span class="p">):</span>
            <span class="n">base_height</span> <span class="o">=</span> <span class="p">(</span><span class="n">max_lr</span> <span class="o">-</span> <span class="n">base_lr</span><span class="p">)</span> <span class="o">*</span> <span class="n">scale_factor</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_mode</span> <span class="o">==</span> <span class="s1">&#39;cycle&#39;</span><span class="p">:</span>
                <span class="n">lr</span> <span class="o">=</span> <span class="n">base_lr</span> <span class="o">+</span> <span class="n">base_height</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_fn</span><span class="p">(</span><span class="n">cycle</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">lr</span> <span class="o">=</span> <span class="n">base_lr</span> <span class="o">+</span> <span class="n">base_height</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_fn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span><span class="p">)</span>
            <span class="n">lrs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">lr</span><span class="p">)</span>

        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cycle_momentum</span><span class="p">:</span>
            <span class="n">momentums</span> <span class="o">=</span> <span class="p">[]</span>
            <span class="k">for</span> <span class="n">base_momentum</span><span class="p">,</span> <span class="n">max_momentum</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">base_momentums</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_momentums</span><span class="p">):</span>
                <span class="n">base_height</span> <span class="o">=</span> <span class="p">(</span><span class="n">max_momentum</span> <span class="o">-</span> <span class="n">base_momentum</span><span class="p">)</span> <span class="o">*</span> <span class="n">scale_factor</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_mode</span> <span class="o">==</span> <span class="s1">&#39;cycle&#39;</span><span class="p">:</span>
                    <span class="n">momentum</span> <span class="o">=</span> <span class="n">max_momentum</span> <span class="o">-</span> <span class="n">base_height</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_fn</span><span class="p">(</span><span class="n">cycle</span><span class="p">)</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">momentum</span> <span class="o">=</span> <span class="n">max_momentum</span> <span class="o">-</span> <span class="n">base_height</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">scale_fn</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span><span class="p">)</span>
                <span class="n">momentums</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">momentum</span><span class="p">)</span>
            <span class="k">for</span> <span class="n">param_group</span><span class="p">,</span> <span class="n">momentum</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">,</span> <span class="n">momentums</span><span class="p">):</span>
                <span class="n">param_group</span><span class="p">[</span><span class="s1">&#39;momentum&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">momentum</span>

        <span class="k">return</span> <span class="n">lrs</span></div></div>


<div class="viewcode-block" id="CosineAnnealingWarmRestarts"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.CosineAnnealingWarmRestarts">[docs]</a><span class="k">class</span> <span class="nc">CosineAnnealingWarmRestarts</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Set the learning rate of each parameter group using a cosine annealing</span>
<span class="sd">    schedule, where :math:`\eta_{max}` is set to the initial lr, :math:`T_{cur}`</span>
<span class="sd">    is the number of epochs since the last restart and :math:`T_{i}` is the number</span>
<span class="sd">    of epochs between two warm restarts in SGDR:</span>

<span class="sd">    .. math::</span>
<span class="sd">        \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1 +</span>
<span class="sd">        \cos\left(\frac{T_{cur}}{T_{i}}\pi\right)\right)</span>

<span class="sd">    When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`.</span>
<span class="sd">    When :math:`T_{cur}=0` after restart, set :math:`\eta_t=\eta_{max}`.</span>

<span class="sd">    It has been proposed in</span>
<span class="sd">    `SGDR: Stochastic Gradient Descent with Warm Restarts`_.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        T_0 (int): Number of iterations for the first restart.</span>
<span class="sd">        T_mult (int, optional): A factor increases :math:`T_{i}` after a restart. Default: 1.</span>
<span class="sd">        eta_min (float, optional): Minimum learning rate. Default: 0.</span>
<span class="sd">        last_epoch (int, optional): The index of last epoch. Default: -1.</span>

<span class="sd">    .. _SGDR\: Stochastic Gradient Descent with Warm Restarts:</span>
<span class="sd">        https://arxiv.org/abs/1608.03983</span>
<span class="sd">    &quot;&quot;&quot;</span>

    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">T_0</span><span class="p">,</span> <span class="n">T_mult</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="n">eta_min</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span> <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>
        <span class="k">if</span> <span class="n">T_0</span> <span class="o">&lt;=</span> <span class="mi">0</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">T_0</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected positive integer T_0, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">T_0</span><span class="p">))</span>
        <span class="k">if</span> <span class="n">T_mult</span> <span class="o">&lt;</span> <span class="mi">1</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">T_mult</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected integer T_mult &gt;= 1, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">T_mult</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">T_0</span> <span class="o">=</span> <span class="n">T_0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">T_i</span> <span class="o">=</span> <span class="n">T_0</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">T_mult</span> <span class="o">=</span> <span class="n">T_mult</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span> <span class="o">=</span> <span class="n">eta_min</span>

        <span class="nb">super</span><span class="p">(</span><span class="n">CosineAnnealingWarmRestarts</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

        <span class="k">return</span> <span class="p">[</span><span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span> <span class="o">+</span> <span class="p">(</span><span class="n">base_lr</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">eta_min</span><span class="p">)</span> <span class="o">*</span> <span class="p">(</span><span class="mi">1</span> <span class="o">+</span> <span class="n">math</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">pi</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_i</span><span class="p">))</span> <span class="o">/</span> <span class="mi">2</span>
                <span class="k">for</span> <span class="n">base_lr</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">base_lrs</span><span class="p">]</span>

<div class="viewcode-block" id="CosineAnnealingWarmRestarts.step"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.CosineAnnealingWarmRestarts.step">[docs]</a>    <span class="k">def</span> <span class="nf">step</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">epoch</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Step could be called after every batch update</span>

<span class="sd">        Example:</span>
<span class="sd">            &gt;&gt;&gt; scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)</span>
<span class="sd">            &gt;&gt;&gt; iters = len(dataloader)</span>
<span class="sd">            &gt;&gt;&gt; for epoch in range(20):</span>
<span class="sd">            &gt;&gt;&gt;     for i, sample in enumerate(dataloader):</span>
<span class="sd">            &gt;&gt;&gt;         inputs, labels = sample[&#39;inputs&#39;], sample[&#39;labels&#39;]</span>
<span class="sd">            &gt;&gt;&gt;         optimizer.zero_grad()</span>
<span class="sd">            &gt;&gt;&gt;         outputs = net(inputs)</span>
<span class="sd">            &gt;&gt;&gt;         loss = criterion(outputs, labels)</span>
<span class="sd">            &gt;&gt;&gt;         loss.backward()</span>
<span class="sd">            &gt;&gt;&gt;         optimizer.step()</span>
<span class="sd">            &gt;&gt;&gt;         scheduler.step(epoch + i / iters)</span>

<span class="sd">        This function can be called in an interleaved way.</span>

<span class="sd">        Example:</span>
<span class="sd">            &gt;&gt;&gt; scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)</span>
<span class="sd">            &gt;&gt;&gt; for epoch in range(20):</span>
<span class="sd">            &gt;&gt;&gt;     scheduler.step()</span>
<span class="sd">            &gt;&gt;&gt; scheduler.step(26)</span>
<span class="sd">            &gt;&gt;&gt; scheduler.step() # scheduler.step(27), instead of scheduler(20)</span>
<span class="sd">        &quot;&quot;&quot;</span>

        <span class="k">if</span> <span class="n">epoch</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
            <span class="n">epoch</span> <span class="o">=</span> <span class="mi">0</span>

        <span class="k">if</span> <span class="n">epoch</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="n">epoch</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">+</span> <span class="mi">1</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">+</span> <span class="mi">1</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_i</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_i</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">T_i</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_i</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_mult</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">epoch</span> <span class="o">&lt;</span> <span class="mi">0</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected non-negative epoch, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">epoch</span><span class="p">))</span>
            <span class="k">if</span> <span class="n">epoch</span> <span class="o">&gt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_0</span><span class="p">:</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_mult</span> <span class="o">==</span> <span class="mi">1</span><span class="p">:</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">=</span> <span class="n">epoch</span> <span class="o">%</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_0</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">n</span> <span class="o">=</span> <span class="nb">int</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">log</span><span class="p">((</span><span class="n">epoch</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_0</span> <span class="o">*</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">T_mult</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">),</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_mult</span><span class="p">))</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">=</span> <span class="n">epoch</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_0</span> <span class="o">*</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">T_mult</span> <span class="o">**</span> <span class="n">n</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span> <span class="o">/</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">T_mult</span> <span class="o">-</span> <span class="mi">1</span><span class="p">)</span>
                    <span class="bp">self</span><span class="o">.</span><span class="n">T_i</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_0</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_mult</span> <span class="o">**</span> <span class="p">(</span><span class="n">n</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">T_i</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">T_0</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">T_cur</span> <span class="o">=</span> <span class="n">epoch</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span> <span class="o">=</span> <span class="n">math</span><span class="o">.</span><span class="n">floor</span><span class="p">(</span><span class="n">epoch</span><span class="p">)</span>

        <span class="k">class</span> <span class="nc">_enable_get_lr_call</span><span class="p">:</span>

            <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">o</span><span class="p">):</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">o</span> <span class="o">=</span> <span class="n">o</span>

            <span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">o</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span> <span class="o">=</span> <span class="kc">True</span>
                <span class="k">return</span> <span class="bp">self</span>

            <span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">type</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">traceback</span><span class="p">):</span>
                <span class="bp">self</span><span class="o">.</span><span class="n">o</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span> <span class="o">=</span> <span class="kc">False</span>
                <span class="k">return</span> <span class="bp">self</span>

        <span class="k">with</span> <span class="n">_enable_get_lr_call</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
            <span class="k">for</span> <span class="n">param_group</span><span class="p">,</span> <span class="n">lr</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">get_lr</span><span class="p">()):</span>
                <span class="n">param_group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">lr</span>

        <span class="bp">self</span><span class="o">.</span><span class="n">_last_lr</span> <span class="o">=</span> <span class="p">[</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;lr&#39;</span><span class="p">]</span> <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">]</span></div></div>


<div class="viewcode-block" id="OneCycleLR"><a class="viewcode-back" href="../../../optim.html#torch.optim.lr_scheduler.OneCycleLR">[docs]</a><span class="k">class</span> <span class="nc">OneCycleLR</span><span class="p">(</span><span class="n">_LRScheduler</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;Sets the learning rate of each parameter group according to the</span>
<span class="sd">    1cycle learning rate policy. The 1cycle policy anneals the learning</span>
<span class="sd">    rate from an initial learning rate to some maximum learning rate and then</span>
<span class="sd">    from that maximum learning rate to some minimum learning rate much lower</span>
<span class="sd">    than the initial learning rate.</span>
<span class="sd">    This policy was initially described in the paper `Super-Convergence:</span>
<span class="sd">    Very Fast Training of Neural Networks Using Large Learning Rates`_.</span>

<span class="sd">    The 1cycle learning rate policy changes the learning rate after every batch.</span>
<span class="sd">    `step` should be called after a batch has been used for training.</span>

<span class="sd">    This scheduler is not chainable.</span>

<span class="sd">    Note also that the total number of steps in the cycle can be determined in one</span>
<span class="sd">    of two ways (listed in order of precedence):</span>

<span class="sd">    #. A value for total_steps is explicitly provided.</span>
<span class="sd">    #. A number of epochs (epochs) and a number of steps per epoch</span>
<span class="sd">       (steps_per_epoch) are provided.</span>
<span class="sd">       In this case, the number of total steps is inferred by</span>
<span class="sd">       total_steps = epochs * steps_per_epoch</span>

<span class="sd">    You must either provide a value for total_steps or provide a value for both</span>
<span class="sd">    epochs and steps_per_epoch.</span>

<span class="sd">    Args:</span>
<span class="sd">        optimizer (Optimizer): Wrapped optimizer.</span>
<span class="sd">        max_lr (float or list): Upper learning rate boundaries in the cycle</span>
<span class="sd">            for each parameter group.</span>
<span class="sd">        total_steps (int): The total number of steps in the cycle. Note that</span>
<span class="sd">            if a value is not provided here, then it must be inferred by providing</span>
<span class="sd">            a value for epochs and steps_per_epoch.</span>
<span class="sd">            Default: None</span>
<span class="sd">        epochs (int): The number of epochs to train for. This is used along</span>
<span class="sd">            with steps_per_epoch in order to infer the total number of steps in the cycle</span>
<span class="sd">            if a value for total_steps is not provided.</span>
<span class="sd">            Default: None</span>
<span class="sd">        steps_per_epoch (int): The number of steps per epoch to train for. This is</span>
<span class="sd">            used along with epochs in order to infer the total number of steps in the</span>
<span class="sd">            cycle if a value for total_steps is not provided.</span>
<span class="sd">            Default: None</span>
<span class="sd">        pct_start (float): The percentage of the cycle (in number of steps) spent</span>
<span class="sd">            increasing the learning rate.</span>
<span class="sd">            Default: 0.3</span>
<span class="sd">        anneal_strategy (str): {&#39;cos&#39;, &#39;linear&#39;}</span>
<span class="sd">            Specifies the annealing strategy: &quot;cos&quot; for cosine annealing, &quot;linear&quot; for</span>
<span class="sd">            linear annealing.</span>
<span class="sd">            Default: &#39;cos&#39;</span>
<span class="sd">        cycle_momentum (bool): If ``True``, momentum is cycled inversely</span>
<span class="sd">            to learning rate between &#39;base_momentum&#39; and &#39;max_momentum&#39;.</span>
<span class="sd">            Default: True</span>
<span class="sd">        base_momentum (float or list): Lower momentum boundaries in the cycle</span>
<span class="sd">            for each parameter group. Note that momentum is cycled inversely</span>
<span class="sd">            to learning rate; at the peak of a cycle, momentum is</span>
<span class="sd">            &#39;base_momentum&#39; and learning rate is &#39;max_lr&#39;.</span>
<span class="sd">            Default: 0.85</span>
<span class="sd">        max_momentum (float or list): Upper momentum boundaries in the cycle</span>
<span class="sd">            for each parameter group. Functionally,</span>
<span class="sd">            it defines the cycle amplitude (max_momentum - base_momentum).</span>
<span class="sd">            Note that momentum is cycled inversely</span>
<span class="sd">            to learning rate; at the start of a cycle, momentum is &#39;max_momentum&#39;</span>
<span class="sd">            and learning rate is &#39;base_lr&#39;</span>
<span class="sd">            Default: 0.95</span>
<span class="sd">        div_factor (float): Determines the initial learning rate via</span>
<span class="sd">            initial_lr = max_lr/div_factor</span>
<span class="sd">            Default: 25</span>
<span class="sd">        final_div_factor (float): Determines the minimum learning rate via</span>
<span class="sd">            min_lr = initial_lr/final_div_factor</span>
<span class="sd">            Default: 1e4</span>
<span class="sd">        last_epoch (int): The index of the last batch. This parameter is used when</span>
<span class="sd">            resuming a training job. Since `step()` should be invoked after each</span>
<span class="sd">            batch instead of after each epoch, this number represents the total</span>
<span class="sd">            number of *batches* computed, not the total number of epochs computed.</span>
<span class="sd">            When last_epoch=-1, the schedule is started from the beginning.</span>
<span class="sd">            Default: -1</span>

<span class="sd">    Example:</span>
<span class="sd">        &gt;&gt;&gt; data_loader = torch.utils.data.DataLoader(...)</span>
<span class="sd">        &gt;&gt;&gt; optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)</span>
<span class="sd">        &gt;&gt;&gt; scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.01, steps_per_epoch=len(data_loader), epochs=10)</span>
<span class="sd">        &gt;&gt;&gt; for epoch in range(10):</span>
<span class="sd">        &gt;&gt;&gt;     for batch in data_loader:</span>
<span class="sd">        &gt;&gt;&gt;         train_batch(...)</span>
<span class="sd">        &gt;&gt;&gt;         scheduler.step()</span>


<span class="sd">    .. _Super-Convergence\: Very Fast Training of Neural Networks Using Large Learning Rates:</span>
<span class="sd">        https://arxiv.org/abs/1708.07120</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                 <span class="n">optimizer</span><span class="p">,</span>
                 <span class="n">max_lr</span><span class="p">,</span>
                 <span class="n">total_steps</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">epochs</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">steps_per_epoch</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
                 <span class="n">pct_start</span><span class="o">=</span><span class="mf">0.3</span><span class="p">,</span>
                 <span class="n">anneal_strategy</span><span class="o">=</span><span class="s1">&#39;cos&#39;</span><span class="p">,</span>
                 <span class="n">cycle_momentum</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
                 <span class="n">base_momentum</span><span class="o">=</span><span class="mf">0.85</span><span class="p">,</span>
                 <span class="n">max_momentum</span><span class="o">=</span><span class="mf">0.95</span><span class="p">,</span>
                 <span class="n">div_factor</span><span class="o">=</span><span class="mf">25.</span><span class="p">,</span>
                 <span class="n">final_div_factor</span><span class="o">=</span><span class="mf">1e4</span><span class="p">,</span>
                 <span class="n">last_epoch</span><span class="o">=-</span><span class="mi">1</span><span class="p">):</span>

        <span class="c1"># Validate optimizer</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">Optimizer</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s1">&#39;</span><span class="si">{}</span><span class="s1"> is not an Optimizer&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                <span class="nb">type</span><span class="p">(</span><span class="n">optimizer</span><span class="p">)</span><span class="o">.</span><span class="vm">__name__</span><span class="p">))</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span> <span class="o">=</span> <span class="n">optimizer</span>

        <span class="c1"># Validate total_steps</span>
        <span class="k">if</span> <span class="n">total_steps</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">epochs</span> <span class="ow">is</span> <span class="kc">None</span> <span class="ow">and</span> <span class="n">steps_per_epoch</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;You must define either total_steps OR (epochs AND steps_per_epoch)&quot;</span><span class="p">)</span>
        <span class="k">elif</span> <span class="n">total_steps</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">total_steps</span> <span class="o">&lt;=</span> <span class="mi">0</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">total_steps</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected non-negative integer total_steps, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">total_steps</span><span class="p">))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">total_steps</span> <span class="o">=</span> <span class="n">total_steps</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">epochs</span> <span class="o">&lt;=</span> <span class="mi">0</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">epochs</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected non-negative integer epochs, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">epochs</span><span class="p">))</span>
            <span class="k">if</span> <span class="n">steps_per_epoch</span> <span class="o">&lt;=</span> <span class="mi">0</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">steps_per_epoch</span><span class="p">,</span> <span class="nb">int</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected non-negative integer steps_per_epoch, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">steps_per_epoch</span><span class="p">))</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">total_steps</span> <span class="o">=</span> <span class="n">epochs</span> <span class="o">*</span> <span class="n">steps_per_epoch</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">step_size_up</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="n">pct_start</span> <span class="o">*</span> <span class="bp">self</span><span class="o">.</span><span class="n">total_steps</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">step_size_down</span> <span class="o">=</span> <span class="nb">float</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">total_steps</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size_up</span><span class="p">)</span> <span class="o">-</span> <span class="mi">1</span>

        <span class="c1"># Validate pct_start</span>
        <span class="k">if</span> <span class="n">pct_start</span> <span class="o">&lt;</span> <span class="mi">0</span> <span class="ow">or</span> <span class="n">pct_start</span> <span class="o">&gt;</span> <span class="mi">1</span> <span class="ow">or</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">pct_start</span><span class="p">,</span> <span class="nb">float</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Expected float between 0 and 1 pct_start, but got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">pct_start</span><span class="p">))</span>

        <span class="c1"># Validate anneal_strategy</span>
        <span class="k">if</span> <span class="n">anneal_strategy</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="s1">&#39;cos&#39;</span><span class="p">,</span> <span class="s1">&#39;linear&#39;</span><span class="p">]:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;anneal_strategy must by one of &#39;cos&#39; or &#39;linear&#39;, instead got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">anneal_strategy</span><span class="p">))</span>
        <span class="k">elif</span> <span class="n">anneal_strategy</span> <span class="o">==</span> <span class="s1">&#39;cos&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">anneal_func</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_annealing_cos</span>
        <span class="k">elif</span> <span class="n">anneal_strategy</span> <span class="o">==</span> <span class="s1">&#39;linear&#39;</span><span class="p">:</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">anneal_func</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_annealing_linear</span>

        <span class="c1"># Initialize learning rate variables</span>
        <span class="n">max_lrs</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_format_param</span><span class="p">(</span><span class="s1">&#39;max_lr&#39;</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">max_lr</span><span class="p">)</span>
        <span class="k">if</span> <span class="n">last_epoch</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
            <span class="k">for</span> <span class="n">idx</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">enumerate</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                <span class="n">group</span><span class="p">[</span><span class="s1">&#39;initial_lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_lrs</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span> <span class="o">/</span> <span class="n">div_factor</span>
                <span class="n">group</span><span class="p">[</span><span class="s1">&#39;max_lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">max_lrs</span><span class="p">[</span><span class="n">idx</span><span class="p">]</span>
                <span class="n">group</span><span class="p">[</span><span class="s1">&#39;min_lr&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;initial_lr&#39;</span><span class="p">]</span> <span class="o">/</span> <span class="n">final_div_factor</span>

        <span class="c1"># Initialize momentum variables</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">cycle_momentum</span> <span class="o">=</span> <span class="n">cycle_momentum</span>
        <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cycle_momentum</span><span class="p">:</span>
            <span class="k">if</span> <span class="s1">&#39;momentum&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">defaults</span> <span class="ow">and</span> <span class="s1">&#39;betas&#39;</span> <span class="ow">not</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">defaults</span><span class="p">:</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s1">&#39;optimizer must support momentum with `cycle_momentum` option enabled&#39;</span><span class="p">)</span>
            <span class="bp">self</span><span class="o">.</span><span class="n">use_beta1</span> <span class="o">=</span> <span class="s1">&#39;betas&#39;</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">defaults</span>
            <span class="n">max_momentums</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_format_param</span><span class="p">(</span><span class="s1">&#39;max_momentum&#39;</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">max_momentum</span><span class="p">)</span>
            <span class="n">base_momentums</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_format_param</span><span class="p">(</span><span class="s1">&#39;base_momentum&#39;</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">base_momentum</span><span class="p">)</span>
            <span class="k">if</span> <span class="n">last_epoch</span> <span class="o">==</span> <span class="o">-</span><span class="mi">1</span><span class="p">:</span>
                <span class="k">for</span> <span class="n">m_momentum</span><span class="p">,</span> <span class="n">b_momentum</span><span class="p">,</span> <span class="n">group</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">max_momentums</span><span class="p">,</span> <span class="n">base_momentums</span><span class="p">,</span> <span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                    <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beta1</span><span class="p">:</span>
                        <span class="n">_</span><span class="p">,</span> <span class="n">beta2</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;betas&#39;</span><span class="p">]</span>
                        <span class="n">group</span><span class="p">[</span><span class="s1">&#39;betas&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">m_momentum</span><span class="p">,</span> <span class="n">beta2</span><span class="p">)</span>
                    <span class="k">else</span><span class="p">:</span>
                        <span class="n">group</span><span class="p">[</span><span class="s1">&#39;momentum&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">m_momentum</span>
                    <span class="n">group</span><span class="p">[</span><span class="s1">&#39;max_momentum&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">m_momentum</span>
                    <span class="n">group</span><span class="p">[</span><span class="s1">&#39;base_momentum&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">b_momentum</span>

        <span class="nb">super</span><span class="p">(</span><span class="n">OneCycleLR</span><span class="p">,</span> <span class="bp">self</span><span class="p">)</span><span class="o">.</span><span class="fm">__init__</span><span class="p">(</span><span class="n">optimizer</span><span class="p">,</span> <span class="n">last_epoch</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_format_param</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">optimizer</span><span class="p">,</span> <span class="n">param</span><span class="p">):</span>
        <span class="sd">&quot;&quot;&quot;Return correctly formatted lr/momentum for each param group.&quot;&quot;&quot;</span>
        <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">param</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
            <span class="k">if</span> <span class="nb">len</span><span class="p">(</span><span class="n">param</span><span class="p">)</span> <span class="o">!=</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">):</span>
                <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;expected </span><span class="si">{}</span><span class="s2"> values for </span><span class="si">{}</span><span class="s2">, got </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                    <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">),</span> <span class="n">name</span><span class="p">,</span> <span class="nb">len</span><span class="p">(</span><span class="n">param</span><span class="p">)))</span>
            <span class="k">return</span> <span class="n">param</span>
        <span class="k">else</span><span class="p">:</span>
            <span class="k">return</span> <span class="p">[</span><span class="n">param</span><span class="p">]</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">)</span>

    <span class="k">def</span> <span class="nf">_annealing_cos</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">,</span> <span class="n">pct</span><span class="p">):</span>
        <span class="s2">&quot;Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0.&quot;</span>
        <span class="n">cos_out</span> <span class="o">=</span> <span class="n">math</span><span class="o">.</span><span class="n">cos</span><span class="p">(</span><span class="n">math</span><span class="o">.</span><span class="n">pi</span> <span class="o">*</span> <span class="n">pct</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span>
        <span class="k">return</span> <span class="n">end</span> <span class="o">+</span> <span class="p">(</span><span class="n">start</span> <span class="o">-</span> <span class="n">end</span><span class="p">)</span> <span class="o">/</span> <span class="mf">2.0</span> <span class="o">*</span> <span class="n">cos_out</span>

    <span class="k">def</span> <span class="nf">_annealing_linear</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">start</span><span class="p">,</span> <span class="n">end</span><span class="p">,</span> <span class="n">pct</span><span class="p">):</span>
        <span class="s2">&quot;Linearly anneal from `start` to `end` as pct goes from 0.0 to 1.0.&quot;</span>
        <span class="k">return</span> <span class="p">(</span><span class="n">end</span> <span class="o">-</span> <span class="n">start</span><span class="p">)</span> <span class="o">*</span> <span class="n">pct</span> <span class="o">+</span> <span class="n">start</span>

    <span class="k">def</span> <span class="nf">get_lr</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_get_lr_called_within_step</span><span class="p">:</span>
            <span class="n">warnings</span><span class="o">.</span><span class="n">warn</span><span class="p">(</span><span class="s2">&quot;To get the last learning rate computed by the scheduler, &quot;</span>
                          <span class="s2">&quot;please use `get_last_lr()`.&quot;</span><span class="p">,</span> <span class="ne">UserWarning</span><span class="p">)</span>

        <span class="n">lrs</span> <span class="o">=</span> <span class="p">[]</span>
        <span class="n">step_num</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_epoch</span>

        <span class="k">if</span> <span class="n">step_num</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">total_steps</span><span class="p">:</span>
            <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Tried to step </span><span class="si">{}</span><span class="s2"> times. The specified number of total steps is </span><span class="si">{}</span><span class="s2">&quot;</span>
                             <span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">step_num</span> <span class="o">+</span> <span class="mi">1</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">total_steps</span><span class="p">))</span>

        <span class="k">for</span> <span class="n">group</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">optimizer</span><span class="o">.</span><span class="n">param_groups</span><span class="p">:</span>
            <span class="k">if</span> <span class="n">step_num</span> <span class="o">&lt;=</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size_up</span><span class="p">:</span>
                <span class="n">computed_lr</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">anneal_func</span><span class="p">(</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;initial_lr&#39;</span><span class="p">],</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;max_lr&#39;</span><span class="p">],</span> <span class="n">step_num</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size_up</span><span class="p">)</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cycle_momentum</span><span class="p">:</span>
                    <span class="n">computed_momentum</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">anneal_func</span><span class="p">(</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;max_momentum&#39;</span><span class="p">],</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;base_momentum&#39;</span><span class="p">],</span>
                                                         <span class="n">step_num</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size_up</span><span class="p">)</span>
            <span class="k">else</span><span class="p">:</span>
                <span class="n">down_step_num</span> <span class="o">=</span> <span class="n">step_num</span> <span class="o">-</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size_up</span>
                <span class="n">computed_lr</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">anneal_func</span><span class="p">(</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;max_lr&#39;</span><span class="p">],</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;min_lr&#39;</span><span class="p">],</span> <span class="n">down_step_num</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size_down</span><span class="p">)</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cycle_momentum</span><span class="p">:</span>
                    <span class="n">computed_momentum</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">anneal_func</span><span class="p">(</span><span class="n">group</span><span class="p">[</span><span class="s1">&#39;base_momentum&#39;</span><span class="p">],</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;max_momentum&#39;</span><span class="p">],</span>
                                                         <span class="n">down_step_num</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">step_size_down</span><span class="p">)</span>

            <span class="n">lrs</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">computed_lr</span><span class="p">)</span>
            <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">cycle_momentum</span><span class="p">:</span>
                <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">use_beta1</span><span class="p">:</span>
                    <span class="n">_</span><span class="p">,</span> <span class="n">beta2</span> <span class="o">=</span> <span class="n">group</span><span class="p">[</span><span class="s1">&#39;betas&#39;</span><span class="p">]</span>
                    <span class="n">group</span><span class="p">[</span><span class="s1">&#39;betas&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="n">computed_momentum</span><span class="p">,</span> <span class="n">beta2</span><span class="p">)</span>
                <span class="k">else</span><span class="p">:</span>
                    <span class="n">group</span><span class="p">[</span><span class="s1">&#39;momentum&#39;</span><span class="p">]</span> <span class="o">=</span> <span class="n">computed_momentum</span>

        <span class="k">return</span> <span class="n">lrs</span></div>
</pre></div>

             </article>
             
            </div>
            <footer>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              
            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="../../../" src="../../../_static/documentation_options.js"></script>
         <script src="../../../_static/jquery.js"></script>
         <script src="../../../_static/underscore.js"></script>
         <script src="../../../_static/doctools.js"></script>
         <script src="../../../_static/language_data.js"></script>
     

  

  <script type="text/javascript" src="../../../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../../../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Stay Connected</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../../../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../../../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Remove any empty p tags that Sphinx adds
      $("[data-tags='null']").remove();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>